#unset NCCL_QUADRUPLE_CHANNELS
#export CUDA_VISIBLE_DEVICES=4,5,1,0,9,8,12,13,7,6,2,3,10,11,15,14
#export CUDA_VISIBLE_DEVICES=4,5,7,6,0,1,3,2,9,8,10,11,13,12,14,15
# export CUDA_VISIBLE_DEVICES=4,7,5,6,1,2,0,3,12,15,13,14,9,10,8,11
# export NCCL_SOCKET_IFNAME=eth0
#export MP_VP=2
sh run_pretrain_megatron_llama.sh  \
dsw  \
/mnt/workspace/ppu_open_source/PPU_ModelZoo/LLM/Pai-Megatron-Patch \
7B   \
1    \
1024 \
3e-4   \
3e-5   \
4096  \
4096  \
8  \
bf16  \
1 \
1 \
sel \
true \
true \
false \
false \
100000 \
/mnt/workspace/datasets/OSCAR/oscar_0_text_document  \
/mnt/workspace/datasets/huggingface/Llama-2-7b-hf \
419430400 \
41943040 \
/mnt/workspace/datasets/huggingface/output/llama2_7B \
true \
true \
None \
false \
